import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from yellowbrick.cluster import KElbowVisualizer
from scipy import stats
from statsmodels.stats import weightstats as st
import numpy as np
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from scipy.cluster.hierarchy import linkage, dendrogram, fcluster
from sklearn.metrics import silhouette_samples, silhouette_score
from factor_analyzer.factor_analyzer import calculate_kmo
import re
import statsmodels.formula.api as smf
from sklearn.metrics import ConfusionMatrixDisplay
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report
from sklearn.preprocessing import OneHotEncoder

raw_data = pd.read_csv("data/segmentation_data.csv")
raw_data.head()

data_no = raw_data[raw_data['Consumi prodotti ittici?']=='No']
len(data_no)

39

no_fish = len(data_no)/len(raw_data)*100
print(f"Percentage of people who don't eat fish (according to our data) is {no_fish:.2f}%")

Percentage of people who don't eat fish (according to our data) is 11.24%

response_distribution = raw_data['Per quale motivo non consumi prodotti ittici?'].value_counts()
response_distribution

Per quale motivo non consumi prodotti ittici?
Questione di gusti                                20
Sono vegetariano/vegano                            7
Hanno un prezzo troppo alto                        3
Questione di gusti;Hanno un prezzo troppo alto     2
Abitudine                                          1
Non so cosa sono i prodotti ittici                 1
Non ci penso non ci faccio caso                    1
Sono vegetariano/vegano;Questione di gusti         1
Sono allergica                                     1
Questione di gusti;                                1
Non mi piacciono                                   1
Name: count, dtype: int64

data = raw_data[raw_data['Consumi prodotti ittici?']=='Si']
data.head()

print(f"Number of people who eat fish products: {len(data)}")
print(f"Percentage: {len(data)/len(raw_data)*100:.2f}%")

Number of people who eat fish products: 308
Percentage: 88.76%

data = data.copy()
data.loc[:, 'respondent_id'] = data.index
data = data.set_index('respondent_id')

segmentation_data = data.iloc[:, 16:28]
segmentation_data

# rename columns
new_names = { 'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La freschezza del prodotto acquistato]' : 'freshness',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La provenienza della materia prima]': 'origin',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La velocità di preparazione]' : 'cooking_speed',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La semplicità di preparazione]' : 'cooking_simplicity',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Il bilancio nutrizionale del prodotto]' : 'nutritional_balance',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La presenza di prodotti nuovi/innovativi]': 'innovative_products',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Che il prodotto sia al naturale/poco lavorato]' : 'unprocessed',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Il prezzo]' : 'price',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La presenza di prodotti monoporzione]' : 'single_portion',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [La provenienza da fonti sostenibili degli ingredienti (es. pesca certificata)]' : 'sustainable_sources',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [Il basso impatto ambientale della produzione]' : 'low_impact_environment',
       'Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [Il packaging eco-sostenibile ]' : 'sustainable_packaging'}

segmentation_data = segmentation_data.rename(columns=new_names)

# map textual responses to numerical values
response_mapping = {
    '1 - Non mi interessa': 1,
    '2 - Poco importante': 2,
    '3 - Mediamente importante': 3,
    '4 - Abbastanza importante': 4,
    '5 - Molto importante': 5
}

for column in segmentation_data.columns:
    segmentation_data[column] = segmentation_data[column].map(response_mapping)

segmentation_data.head()

segmentation_data.info()

<class 'pandas.core.frame.DataFrame'>
Index: 308 entries, 0 to 345
Data columns (total 12 columns):
 #   Column                  Non-Null Count  Dtype
---  ------                  --------------  -----
 0   freshness               308 non-null    int64
 1   origin                  308 non-null    int64
 2   cooking_speed           308 non-null    int64
 3   cooking_simplicity      308 non-null    int64
 4   nutritional_balance     308 non-null    int64
 5   innovative_products     308 non-null    int64
 6   unprocessed             308 non-null    int64
 7   price                   308 non-null    int64
 8   single_portion          308 non-null    int64
 9   sustainable_sources     308 non-null    int64
 10  low_impact_environment  308 non-null    int64
 11  sustainable_packaging   308 non-null    int64
dtypes: int64(12)
memory usage: 31.3 KB

segmentation_data.describe().style.format("{:.1f}")

num_variables = len(segmentation_data.columns) 
num_rows = num_variables // 3 + (num_variables % 3 > 0)

fig, axes = plt.subplots(nrows=num_rows, ncols=3, figsize=(15, num_rows * 4), tight_layout=True)
axes = axes.flatten()

for index, column in enumerate(segmentation_data.columns):
    axes[index].hist(segmentation_data[column], bins=5, range=(1, 5), alpha=0.7, color='blue', edgecolor='black')
    axes[index].set_title(column)
    axes[index].set_xlabel('Rating')
    axes[index].set_ylabel('Frequency')
    axes[index].set_xticks(range(1, 6))

for i in range(index + 1, len(axes)):
    axes[i].axis('off')

plt.show()

sns.set(style="whitegrid")
df_melted = segmentation_data.melt(var_name='Variable', value_name='Value')

plt.figure(figsize=(12, 6))
sns.boxplot(x='Variable', y='Value', data=df_melted, hue='Variable', palette='Set2', legend=False)
means = segmentation_data.mean()

for i, mean in enumerate(means):
    plt.scatter(i, mean, color='black', label='Mean' if i == 0 else "", s=100, zorder=10)
    
plt.title('Boxplots of Each Variable', fontsize=14, fontweight='bold')
plt.xlabel('')
plt.ylabel('Value')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

corr_matrix = segmentation_data.corr()

plt.figure(figsize=(10, 8))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', linewidths=0.5)
plt.title('Correlation matrix')
plt.show()

segmentation_data['speed_n_simplicity'] = segmentation_data[['cooking_speed', 'cooking_simplicity']].mean(axis=1)
final_segmentation_data = segmentation_data.drop(['cooking_speed', 'cooking_simplicity', 'low_impact_environment'], axis=1)
final_segmentation_data.head()

corr_matrix = final_segmentation_data.corr()

plt.figure(figsize=(10, 8))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', linewidths=0.5)
plt.title('Correlation matrix')
plt.show()

model = KMeans()
visualizer = KElbowVisualizer(model,timings=False, locate_elbow= False)
visualizer.fit(final_segmentation_data)
visualizer.show();

visualizer = KElbowVisualizer(model,metric='silhouette',timings=False, locate_elbow= False)
visualizer.fit(final_segmentation_data)
visualizer.show();

# K-Means and assign clusters to data
kmeans2 = KMeans(n_clusters = 2, random_state=42)
kmeans2.fit(final_segmentation_data)
final_segmentation_data_2 = final_segmentation_data.copy()
final_segmentation_data_2['Cluster_KMeans'] = kmeans2.labels_ + 1

# Silhouette Plot
cluster_labels = kmeans2.fit_predict(final_segmentation_data_2)
silhouette_vals = silhouette_samples(final_segmentation_data_2, cluster_labels)
silhouette_avg = silhouette_score(final_segmentation_data_2, cluster_labels)
n_clusters = 2
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10  # 10 for spacing between clusters

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.27171684341213087

# Compute Clusters' size
cluster_sizes = final_segmentation_data_2['Cluster_KMeans'].value_counts()
cluster_percentages = (cluster_sizes / cluster_sizes.sum()) * 100

cluster_summary = pd.DataFrame({
    'Cluster': cluster_sizes.index,
    'Count': cluster_sizes.values,
    'Percentage': cluster_percentages.values
})

cluster_summary = cluster_summary.sort_values(by='Cluster').reset_index(drop=True)
cluster_summary

# Compute centroids' means
centroids2 = final_segmentation_data_2.groupby("Cluster_KMeans").mean().T
centroids2.insert(0,"Population",final_segmentation_data_2.mean())
centroids2.round(2)

X = final_segmentation_data_2.copy()

def style_cols(col):
    if col.name == "Population": return ["background-color: white"]*len(col)
    
    segment = X[X.Cluster_KMeans == col.name]
    rest_pop = X[X.Cluster_KMeans != col.name]
    
    colors = []
    for var_name, var_segment_mean in col.items():
        t_test, p_value = stats.ttest_ind(segment[var_name], rest_pop[var_name],equal_var=False)
        #_,p_value,_ = st.ttest_ind(segment[var_name],rest_pop[var_name],usevar="unequal")    
        if p_value < 0.05:
            color = "background-color: green" if var_segment_mean > rest_pop[var_name].mean() else "background-color: red"
        elif p_value < 0.1:
            color = "background-color: #90ee90" if var_segment_mean > rest_pop[var_name].mean() else "background-color: #f1807e"
        else: color = "background-color: white"
        colors.append(color)
    return colors

centroids2.style.apply(style_cols).format("{:.2f}")

# Compute KMeans and assign clusters to data
kmeans4 = KMeans(n_clusters = 4, random_state=42)
kmeans4.fit(final_segmentation_data)
final_segmentation_data_4 = final_segmentation_data.copy()
final_segmentation_data_4['Cluster_KMeans'] = kmeans4.labels_ +1
final_segmentation_data_4

# Silhouette Plot
cluster_labels = kmeans4.fit_predict(final_segmentation_data_4)
silhouette_vals = silhouette_samples(final_segmentation_data_4, cluster_labels)
silhouette_avg = silhouette_score(final_segmentation_data_4, cluster_labels)
n_clusters = 4
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.22220700779410574

# Compute Clusters' size
cluster_sizes = final_segmentation_data_4['Cluster_KMeans'].value_counts()
cluster_percentages = (cluster_sizes / cluster_sizes.sum()) * 100

cluster_summary = pd.DataFrame({
    'Cluster': cluster_sizes.index,
    'Count': cluster_sizes.values,
    'Percentage': cluster_percentages.values
})

cluster_summary = cluster_summary.sort_values(by='Cluster').reset_index(drop=True)
cluster_summary

# Compute centroids means
centroids4 = final_segmentation_data_4.groupby("Cluster_KMeans").mean().T
centroids4.insert(0,"Population",final_segmentation_data_4.mean())
centroids4.round(2)

# Compute means table
X = final_segmentation_data_4.copy()
centroids4.style.apply(style_cols).format("{:.2f}")

# Hierarchical Clustering
Z = linkage(final_segmentation_data, method='ward')

plt.figure(figsize=(10, 5))
dendrogram(Z, leaf_rotation=90, leaf_font_size=8)
plt.title('Hierarchical Clustering Dendogram')
plt.xlabel('Observations')
plt.ylabel('Distance')
plt.show()

# Silhouette score
silhouette_scores = []
cluster_range = range(2, 10) 

for k in cluster_range:
    labels = fcluster(Z, k, criterion='maxclust')
    score = silhouette_score(final_segmentation_data, labels)
    silhouette_scores.append(score)
    
plt.figure(figsize=(8, 6))
plt.plot(cluster_range, silhouette_scores, marker='o', linestyle='-', color='b')
plt.title('Silhouette Score vs Number of Clusters')
plt.xlabel('Number of Clusters (k)')
plt.ylabel('Silhouette Score')
plt.xticks(cluster_range)
plt.grid(True)
plt.show()

kmo_all, kmo_model = calculate_kmo(final_segmentation_data)
print(f"KMO Value: {kmo_model}")

KMO Value: 0.8457551794181936

# PCA
pca_full = PCA(n_components=len(final_segmentation_data.columns))
principal_components_full = pca_full.fit_transform(final_segmentation_data)
explained_variance = pca_full.explained_variance_ratio_
explained_variance_percent = explained_variance * 100 
cumulative_variance = explained_variance.cumsum()

plt.figure(figsize=(8,4))
plt.bar(range(1, len(explained_variance)+1), explained_variance, alpha=0.7)
plt.step(range(1, len(cumulative_variance)+1), cumulative_variance, where='mid', color='orange')
plt.ylabel('Explained Variance Ratio')
plt.xlabel('Principal Components')
plt.title('PCA Explained Variance')
plt.show()

# PCA with 5 components
pca = PCA(n_components=5)
principal_components = pca.fit_transform(final_segmentation_data)

# Create a DataFrame with the 5 principal components
pca_df = pd.DataFrame(principal_components, 
                      columns=['PC1','PC2','PC3','PC4','PC5'])

# Explained variance ratios for the 5 PCs
explained_variance = pca.explained_variance_ratio_
cumulative_variance = explained_variance.cumsum()


# Plot explained variance
plt.figure(figsize=(8,4))
plt.bar(range(1, len(explained_variance)+1), explained_variance, alpha=0.7)
plt.step(range(1, len(cumulative_variance)+1), cumulative_variance, 
         where='mid', color='orange')
plt.ylabel('Explained Variance Ratio')
plt.xlabel('Principal Components')
plt.title('PCA Explained Variance (5 Components)')
plt.show()

print("Explained variance per component:", explained_variance)
print("Cumulative explained variance:", cumulative_variance)

Explained variance per component: [0.43791451 0.1532498  0.08571788 0.07275902 0.06489671]
Cumulative explained variance: [0.43791451 0.59116432 0.67688219 0.74964121 0.81453792]

loadings = pd.DataFrame(
    pca.components_.T, 
    columns=[f'PC{i+1}' for i in range(pca.n_components_)],
    index=final_segmentation_data.columns
)

print("PCA Loadings (Coefficients):")
print(loadings)

for pc in loadings.columns:
    # Sort by absolute value to find highest magnitude (positive or negative)
    sorted_loadings = loadings[pc].abs().sort_values(ascending=False)
    top_features = sorted_loadings.head(5) 
    print(f"\nTop loadings for {pc}:")
    print(top_features)

PCA Loadings (Coefficients):
                            PC1       PC2       PC3       PC4       PC5
freshness              0.290192 -0.277332 -0.381556  0.009444 -0.352309
origin                 0.374445 -0.294280 -0.107446  0.140588 -0.364213
nutritional_balance    0.363140  0.078082 -0.061512  0.193274  0.635066
innovative_products    0.277948  0.319165  0.315480  0.674411 -0.047230
unprocessed            0.336029 -0.150646 -0.274893  0.260338  0.010167
price                  0.241113  0.224056 -0.415531 -0.125268  0.279361
single_portion         0.289727  0.651848  0.158921 -0.211251 -0.459084
sustainable_sources    0.357426 -0.317366  0.272023 -0.249371 -0.028263
sustainable_packaging  0.365565 -0.198513  0.551036 -0.340999  0.196881
speed_n_simplicity     0.223762  0.300989 -0.297378 -0.425910  0.096036

Top loadings for PC1:
origin                   0.374445
sustainable_packaging    0.365565
nutritional_balance      0.363140
sustainable_sources      0.357426
unprocessed              0.336029
Name: PC1, dtype: float64

Top loadings for PC2:
single_portion         0.651848
innovative_products    0.319165
sustainable_sources    0.317366
speed_n_simplicity     0.300989
origin                 0.294280
Name: PC2, dtype: float64

Top loadings for PC3:
sustainable_packaging    0.551036
price                    0.415531
freshness                0.381556
innovative_products      0.315480
speed_n_simplicity       0.297378
Name: PC3, dtype: float64

Top loadings for PC4:
innovative_products      0.674411
speed_n_simplicity       0.425910
sustainable_packaging    0.340999
unprocessed              0.260338
sustainable_sources      0.249371
Name: PC4, dtype: float64

Top loadings for PC5:
nutritional_balance    0.635066
single_portion         0.459084
origin                 0.364213
freshness              0.352309
price                  0.279361
Name: PC5, dtype: float64

inertias = []
K_range = range(1, 11)

for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(pca_df)
    inertias.append(kmeans.inertia_)

plt.figure(figsize=(8,5))
plt.plot(K_range, inertias, 'bo-')
plt.title("Elbow Method for K-Means")
plt.xlabel("Number of clusters (k)")
plt.ylabel("Inertia (Sum of Squared Distances)")
plt.show()

if 'Cluster' in pca_df.columns:
    pca_df.drop('Cluster', axis=1, inplace=True)
    
# Fit K-Means
k=4
kmeans = KMeans(n_clusters=k, random_state=42)
kmeans.fit(pca_df[['PC1','PC2', 'PC3', 'PC4', 'PC5']]) 

pca_df['Cluster'] = kmeans.labels_ + 1

# Plot each cluster with a different color
colors = ['red', 'green', 'blue', 'magenta', 'orange']
plt.figure(figsize=(8,6))

for cluster_label in range(1, k + 1):
    subset = pca_df[pca_df['Cluster'] == cluster_label]
    plt.scatter(
        subset['PC1'], subset['PC2'], 
        s=100, 
        c=colors[cluster_label - 1], 
        label=f'Cluster {cluster_label}'
    )

plt.xlabel('Principal Component 1')
plt.ylabel('Principal Component 2')
plt.title('K-Means Clusters (PCA-Reduced Data)')
plt.legend()
plt.show()

score = silhouette_score(pca_df[['PC1','PC2','PC3','PC4','PC5']], pca_df['Cluster'])
print("Silhouette Score:", score)

Silhouette Score: 0.22150356675028918

final_segmentation_data['Cluster'] = pca_df['Cluster']

# Compute mean of original variables by cluster
cluster_means = final_segmentation_data.groupby('Cluster').mean()
print("Cluster Means:")
print(cluster_means)

Cluster Means:
         freshness    origin  nutritional_balance  innovative_products  \
Cluster                                                                  
1.0       4.333333  3.863636             3.666667             2.636364   
2.0       4.406593  3.813187             3.670330             2.681319   
3.0       4.375000  4.138889             3.819444             2.736111   
4.0       3.930233  3.511628             3.534884             2.279070   

         unprocessed     price  single_portion  sustainable_sources  \
Cluster                                                               
1.0         3.863636  4.015152        2.818182             3.818182   
2.0         3.934066  3.934066        2.747253             3.890110   
3.0         4.069444  3.861111        2.708333             4.000000   
4.0         3.790698  3.697674        2.720930             3.511628   

         sustainable_packaging  speed_n_simplicity  
Cluster                                             
1.0                   3.515152            3.265152  
2.0                   3.758242            3.274725  
3.0                   3.944444            3.381944  
4.0                   3.093023            3.348837

if 'Cluster' in pca_df.columns:
    pca_df.drop('Cluster', axis=1, inplace=True)
    
# Fit K-Means
k=2
kmeans = KMeans(n_clusters=k, random_state=42)
kmeans.fit(pca_df[['PC1','PC2', 'PC3', 'PC4', 'PC5']])

pca_df['Cluster'] = kmeans.labels_ + 1

# Plot each cluster with a different color
colors = ['red', 'green']
plt.figure(figsize=(8,6))

for cluster_label in range(1, k + 1):
    subset = pca_df[pca_df['Cluster'] == cluster_label]
    plt.scatter(
        subset['PC1'], subset['PC2'], 
        s=100, 
        c=colors[cluster_label - 1], 
        label=f'Cluster {cluster_label}'
    )

plt.xlabel('Principal Component 1')
plt.ylabel('Principal Component 2')
plt.title('K-Means Clusters (PCA-Reduced Data)')
plt.legend()
plt.show()

score = silhouette_score(pca_df[['PC1','PC2','PC3','PC4','PC5']], pca_df['Cluster'])
print("Silhouette Score:", score)

Silhouette Score: 0.3052997283228182

final_segmentation_data['Cluster'] = pca_df['Cluster']

# Compute mean of original variables by cluster
cluster_means = final_segmentation_data.groupby('Cluster').mean()
print("Cluster Means:")
print(cluster_means)

Cluster Means:
         freshness    origin  nutritional_balance  innovative_products  \
Cluster                                                                  
1.0       4.355828  3.975460             3.791411             2.730061   
2.0       4.229358  3.697248             3.532110             2.458716   

         unprocessed     price  single_portion  sustainable_sources  \
Cluster                                                               
1.0         4.018405  3.938650        2.730061             3.987730   
2.0         3.798165  3.834862        2.779817             3.623853   

         sustainable_packaging  speed_n_simplicity  
Cluster                                             
1.0                   3.828221            3.319018  
2.0                   3.366972            3.302752

reduced_data = final_segmentation_data[['innovative_products', 'price', 'single_portion', 'speed_n_simplicity']]
reduced_data.head()

num_variables = len(reduced_data.columns) 
num_rows = num_variables // 3 + (num_variables % 3 > 0)

fig, axes = plt.subplots(nrows=num_rows, ncols=3, figsize=(15, num_rows * 4), tight_layout=True)
axes = axes.flatten()

for index, column in enumerate(reduced_data.columns):
    axes[index].hist(reduced_data[column], bins=5, range=(1, 5), alpha=0.7, color='blue', edgecolor='black')
    axes[index].set_title(column)
    axes[index].set_xlabel('Rating')
    axes[index].set_ylabel('Frequency')
    axes[index].set_xticks(range(1, 6))

for i in range(index + 1, len(axes)):
    axes[i].axis('off')

plt.show()

model = KMeans()
visualizer = KElbowVisualizer(model,timings=False, locate_elbow= False)
visualizer.fit(reduced_data)
visualizer.show();

visualizer = KElbowVisualizer(model,metric='silhouette',timings=False, locate_elbow= False)
visualizer.fit(reduced_data)
visualizer.show();

# K-Means and assign clusters to data
reduced_data = reduced_data.copy()
kmeans2 = KMeans(n_clusters = 2, random_state=42)
kmeans2.fit(reduced_data)
reduced_data['Cluster_K2'] = kmeans2.labels_ + 1

# Silhouette Plot
cluster_labels = kmeans2.fit_predict(reduced_data)
silhouette_vals = silhouette_samples(reduced_data, cluster_labels)
silhouette_avg = silhouette_score(reduced_data, cluster_labels)
n_clusters = 2
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10  # 10 for spacing between clusters

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.32039644786649446

# Compute KMeans and assign clusters to data
reduced_data.drop(['Cluster_K2'], axis=1, inplace=True)
kmeans4 = KMeans(n_clusters = 4, random_state=42)
kmeans4.fit(reduced_data)
reduced_data['Cluster_K4'] = kmeans4.labels_ + 1

# Silhouette Plot
cluster_labels = kmeans4.fit_predict(reduced_data)
silhouette_vals = silhouette_samples(reduced_data, cluster_labels)
silhouette_avg = silhouette_score(reduced_data, cluster_labels)
n_clusters = 4
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10  # 10 for spacing between clusters

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.36025049701924083

reduced_data.drop(['Cluster_K4'], axis=1, inplace=True)
reduced_data2 =  final_segmentation_data[['innovative_products', 'single_portion', 'speed_n_simplicity']]
reduced_data2.head()

model = KMeans()
visualizer = KElbowVisualizer(model,timings=False, locate_elbow= False)
visualizer.fit(reduced_data2)
visualizer.show();

visualizer = KElbowVisualizer(model,metric='silhouette',timings=False, locate_elbow= False)
visualizer.fit(reduced_data2)
visualizer.show();

# K-Means and assign clusters to data
reduced_data2 = reduced_data2.copy()
kmeans2 = KMeans(n_clusters = 2, random_state=42)
kmeans2.fit(reduced_data2)
reduced_data2['Cluster_K2'] = kmeans2.labels_ + 1

# Silhouette Plot
cluster_labels = kmeans2.fit_predict(reduced_data2)
silhouette_vals = silhouette_samples(reduced_data2, cluster_labels)
silhouette_avg = silhouette_score(reduced_data2, cluster_labels)
n_clusters = 2
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10  # 10 for spacing between clusters

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.3746443292384108

# Compute KMeans and assign clusters to data
reduced_data2.drop(['Cluster_K2'], axis=1, inplace=True)
kmeans4 = KMeans(n_clusters = 4, random_state=42)
kmeans4.fit(reduced_data2)
reduced_data2['Cluster_K4'] = kmeans4.labels_ + 1

# Silhouette Plot
cluster_labels = kmeans4.fit_predict(reduced_data2)
silhouette_vals = silhouette_samples(reduced_data2, cluster_labels)
silhouette_avg = silhouette_score(reduced_data2, cluster_labels)
n_clusters = 4
y_lower = 10
plt.figure(figsize=(10, 6))
for i in range(n_clusters):
    ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
    ith_cluster_silhouette_values.sort()
    size_cluster_i = ith_cluster_silhouette_values.shape[0]
    y_upper = y_lower + size_cluster_i

    plt.fill_betweenx(np.arange(y_lower, y_upper),
                      0, ith_cluster_silhouette_values,
                      alpha=0.7)
    plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
    y_lower = y_upper + 10  # 10 for spacing between clusters

plt.axvline(x=silhouette_avg, color="red", linestyle="--")
plt.title("Silhouette Plot for KMeans Clustering")
plt.xlabel("Silhouette Coefficient")
plt.ylabel("Cluster")
plt.show()

print("Silhouette Score:", silhouette_avg)

Silhouette Score: 0.40194765380996833

descriptor_data = data.iloc[:,1:16]
descriptor_data = descriptor_data.drop(descriptor_data.columns[[6,7,14]], axis=1)

# rename columns
new_names = { 'Qual è la tua età?': 'age', 
              'In che regione vivi?':'region', 
              'In che zona abiti?': 'location',
       'Con quale genere ti identifichi?' : 'gender',
       'Da quanti componenti è formato il tuo nucleo familiare?' : 'family_members',
       'Qual è la tua professione?': 'job',
        'Quanto spesso consumi prodotti ittici?': 'consumption_frequency',
       'Che tipo di prodotti ittici consumi più frequentemente in casa?': 'fish_type',
       'Quanto tempo dedichi mediamente alla preparazione dei pasti durante la giornata?' : 'meal_preparation_time',
       'Solitamente ti occupi tu della spesa nella tua famiglia?': 'grocery_responsibility',
       'Dove acquisti più spesso i tuoi prodotti ittici? (Puoi selezionare più opzioni) ': 'grocery_place',
       'Quanto saresti disposto a spendere per un prodotto ittico di alta qualità (es. burger di pesce gourmet)?': 'WTP_fish'
    
}

descriptor_data = descriptor_data.rename(columns=new_names)
descriptor_data

# remove emoji
def remove_emojis(text):
    return re.sub(r'[\U00010000-\U0010ffff]', '', text)

descriptor_data = descriptor_data.map(lambda x: remove_emojis(x) if isinstance(x, str) else x)

# Plot bar chart for each variable
sns.set(style="whitegrid")

def plot_bar_grid(df, columns, cols=1, explode_columns=None, delimiter=','):
    if explode_columns is None:
        explode_columns = []

    processed_df = df.copy()
    for column in explode_columns:
        if column in processed_df.columns:
            processed_df = processed_df.dropna(subset=[column])
            processed_df[column] = processed_df[column].str.split(delimiter)
            processed_df = processed_df.explode(column)

    total_plots = len(columns)
    rows = (total_plots + cols - 1) // cols 

    fig, axes = plt.subplots(rows, cols, figsize=(20, rows * 6))  
    axes = axes.flatten()

    for i, column in enumerate(columns):
        if column in processed_df.columns:
            sns.countplot(data=processed_df, x=column, hue=column, palette='Set2', legend=False, ax=axes[i])
            axes[i].set_title(f'Distribution of {column}', fontsize=17)
            axes[i].tick_params(axis='x', rotation=30, labelsize=13) 
            axes[i].tick_params(axis='y', labelsize=13) 

    for j in range(len(columns), len(axes)):
        fig.delaxes(axes[j])

    plt.tight_layout()
    plt.subplots_adjust(hspace=1) 
    plt.show()


columns_to_plot = descriptor_data.columns
plot_bar_grid(
    descriptor_data,
    columns_to_plot,
    cols=1, 
    explode_columns=['grocery_place'], 
    delimiter=';' 
)

descriptor_data['Cluster_K2'] = final_segmentation_data_2['Cluster_KMeans']
descriptor_data['Cluster_K4'] = final_segmentation_data_4['Cluster_KMeans']

clusters = descriptor_data['Cluster_K2'].unique()
variables = [col for col in descriptor_data.columns if col not in ['Cluster_K2', 'respondent_id']]

num_clusters = len(clusters)
num_variables = len(variables)
fig, axs = plt.subplots(num_clusters, num_variables, figsize=(5 * num_variables, 5 * num_clusters)) 

if num_clusters == 1 or num_variables == 1:
    axs = np.array(axs).reshape(num_clusters, num_variables)

# Plot pie charts for each cluster and variable
for i, cluster in enumerate(clusters):
    cluster_data = descriptor_data[descriptor_data['Cluster_K2'] == cluster]
    for j, var in enumerate(variables):
        counts = cluster_data[var].value_counts()
        ax = axs[i, j] if num_clusters > 1 else axs[j]
        ax.pie(counts, labels=counts.index, autopct='%1.1f%%', startangle=90)
        ax.set_title(f'Cluster {cluster} - {var}')

plt.tight_layout()
plt.show()

C:\Users\Pancione\AppData\Local\Temp\ipykernel_10304\3132827631.py:20: UserWarning: Tight layout not applied. tight_layout cannot make Axes width small enough to accommodate all Axes decorations
  plt.tight_layout()

clusters = descriptor_data['Cluster_K4'].unique()
variables = [col for col in descriptor_data.columns if col not in ['Cluster_K4', 'respondent_id']]

num_clusters = len(clusters)
num_variables = len(variables)
fig, axs = plt.subplots(num_clusters, num_variables, figsize=(5 * num_variables, 5 * num_clusters))

if num_clusters == 1 or num_variables == 1:
    axs = np.array(axs).reshape(num_clusters, num_variables)

# Plot pie charts for each cluster and variable
for i, cluster in enumerate(clusters):
    cluster_data = descriptor_data[descriptor_data['Cluster_K4'] == cluster]
    for j, var in enumerate(variables): 
        counts = cluster_data[var].value_counts()
        ax = axs[i, j] if num_clusters > 1 else axs[j]
        ax.pie(counts, labels=counts.index, autopct='%1.1f%%', startangle=90)
        ax.set_title(f'Cluster {cluster} - {var}')

plt.tight_layout()
plt.show()

C:\Users\Pancione\AppData\Local\Temp\ipykernel_10304\977047102.py:20: UserWarning: Tight layout not applied. tight_layout cannot make Axes width small enough to accommodate all Axes decorations
  plt.tight_layout()

df = descriptor_data[['age', 'region', 'location', 'gender', 'family_members', 'job', 'Cluster_K4']].copy()
df["location"] = df["location"].replace("Esterno (es. campagna/costiera/montagna...)", "Esterno")
df

print((df['gender'] == 'Altro').sum())

2

df_logit = df[df['gender'] != 'Altro']
df_logit

category_counts = df_logit.nunique()
category_counts

age                6
region            15
location           3
gender             2
family_members     3
job                4
Cluster_K4         4
dtype: int64

df_reduced = df_logit[['age', 'location', 'gender', 'family_members', 'job', 'Cluster_K4']]
df_reduced= pd.get_dummies(df_reduced, drop_first=True, dtype=int)

# renaming columns
df_reduced.columns = df_reduced.columns.str.replace(r"[^\w]", "_", regex=True)

df_vif = df_reduced.drop(columns=['Cluster_K4'])
vif_data = pd.DataFrame()
vif_data["Feature"] = df_vif.columns
vif_data["VIF"] = [variance_inflation_factor(df_vif.values, i) for i in range(df_vif.shape[1])]

print(vif_data)

                     Feature       VIF
0             age_26_35_anni  1.556625
1             age_36_45_anni  1.195499
2             age_46_60_anni  2.226075
3             age_60_70_anni  1.985012
4               age_70__anni  1.586472
5           location_Esterno  1.717330
6   location_Periferia_città  2.280231
7                gender_Uomo  1.553919
8         family_members_2_3  4.179946
9          family_members_4_  4.283334
10            job_Lavoratore  5.785798
11            job_Pensionato  2.414007
12              job_Studente  3.467430

independent_vars = df_reduced.columns.difference(['Cluster_K4']).tolist()
formula = "Cluster_K4 ~ " + " + ".join(independent_vars)

model1 = smf.mnlogit(formula=formula, data=df_reduced).fit()
model1.summary()

Warning: Maximum number of iterations has been exceeded.
         Current function value: 1.247734
         Iterations: 35

C:\Users\Pancione\AppData\Local\Programs\Python\Python312\Lib\site-packages\statsmodels\base\model.py:607: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals
  warnings.warn("Maximum Likelihood optimization failed to "

df_reduced = df[['gender', 'location', 'family_members', 'job', 'Cluster_K4']]
df_reduced = pd.get_dummies(df_reduced, drop_first=True, dtype=int)

# renaming columns
df_reduced.columns = df_reduced.columns.str.replace(r"[^\w]", "_", regex=True)

# fitting model
independent_vars = df_reduced.columns.difference(['Cluster_K4']).tolist()
formula = "Cluster_K4 ~ " + " + ".join(independent_vars)

model2 = smf.mnlogit(formula=formula, data=df_reduced).fit()
model2.summary()

Warning: Maximum number of iterations has been exceeded.
         Current function value: 1.258016
         Iterations: 35

C:\Users\Pancione\AppData\Local\Programs\Python\Python312\Lib\site-packages\statsmodels\base\model.py:607: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals
  warnings.warn("Maximum Likelihood optimization failed to "

df_reduced = df[['gender', 'family_members', 'job', 'Cluster_K4']]
df_reduced = pd.get_dummies(df_reduced, drop_first=True, dtype=int)

# renaming columns
df_reduced.columns = df_reduced.columns.str.replace(r"[^\w]", "_", regex=True)

# fitting model
independent_vars = df_reduced.columns.difference(['Cluster_K4']).tolist()
formula = "Cluster_K4 ~ " + " + ".join(independent_vars)

model3 = smf.mnlogit(formula=formula, data=df_reduced).fit()
model3.summary()

Warning: Maximum number of iterations has been exceeded.
         Current function value: 1.268925
         Iterations: 35

C:\Users\Pancione\AppData\Local\Programs\Python\Python312\Lib\site-packages\statsmodels\base\model.py:607: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals
  warnings.warn("Maximum Likelihood optimization failed to "

PredTable = model1.pred_table()
cluster_labels = np.arange(1, PredTable.shape[0] + 1)
ConfusionMatrixDisplay(confusion_matrix=PredTable, display_labels=cluster_labels).plot()

plt.title("Confusion Matrix")
plt.show()

# apply One-Hot Encoding to categorical variables
df_encoded = pd.get_dummies(df, columns=["age", "region", "location", "gender", "family_members", "job"], drop_first=True)

X = df_encoded.drop(columns=["Cluster_K4"])
y = df_encoded["Cluster_K4"]

# split dataset into training and test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Random Forest model
rf = RandomForestClassifier(n_estimators=30, random_state=42)
rf.fit(X_train, y_train)

# predictions
y_pred = rf.predict(X_test)

# model accuracy
accuracy = accuracy_score(y_test, y_pred)
accuracy

0.3709677419354839

# classification report
report = classification_report(y_test, y_pred, output_dict=True)
report_df = pd.DataFrame(report).transpose()
print(report_df)

              precision    recall  f1-score    support
1              0.250000  0.285714  0.266667  14.000000
2              0.750000  0.214286  0.333333  14.000000
3              0.461538  0.400000  0.428571  15.000000
4              0.344828  0.526316  0.416667  19.000000
accuracy       0.370968  0.370968  0.370968   0.370968
macro avg      0.451592  0.356579  0.361310  62.000000
weighted avg   0.443142  0.370968  0.366859  62.000000

feature_importances = rf.feature_importances_
feature_importance_df = pd.DataFrame({'Feature': X.columns, 'Importance': feature_importances})
feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)

plt.figure(figsize=(10,8))
plt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'])
plt.xlabel('Feature Importance')
plt.ylabel('Feature')
plt.title('Feature importance')
plt.gca().invert_yaxis()
plt.show()

# Set threshold for feature importance (remove features with importance < 0.01)
threshold = 0.01
important_features = feature_importance_df[feature_importance_df['Importance'] >= threshold]['Feature']
X_reduced = X[important_features]

# split dataset into training and test set
X_train_red, X_test_red, y_train_red, y_test_red = train_test_split(X_reduced, y, test_size=0.2, random_state=42)

# retrain the Random Forest model with reduced features
rf_reduced = RandomForestClassifier(n_estimators=30, random_state=42)
rf_reduced.fit(X_train_red, y_train_red)

# predictions
y_pred_red = rf_reduced.predict(X_test_red)

# accuracy
accuracy_reduced = accuracy_score(y_test_red, y_pred_red)
print(accuracy_reduced)

# classification report
report_reduced = classification_report(y_test_red, y_pred_red, output_dict=True)
report_reduced_df = pd.DataFrame(report_reduced).transpose()
print(report_reduced_df)

0.3387096774193548
              precision    recall  f1-score   support
1              0.200000  0.214286  0.206897  14.00000
2              0.750000  0.214286  0.333333  14.00000
3              0.461538  0.400000  0.428571  15.00000
4              0.300000  0.473684  0.367347  19.00000
accuracy       0.338710  0.338710  0.338710   0.33871
macro avg      0.427885  0.325564  0.334037  62.00000
weighted avg   0.418114  0.338710  0.338248  62.00000

Cluster_KMeans	Population	1	2	3	4
freshness	4.29	4.13	2.90	4.76	4.75
origin	3.88	3.28	2.43	4.79	4.39
nutritional_balance	3.69	3.52	2.18	4.67	3.85
innovative_products	2.60	2.59	1.69	3.76	2.20
unprocessed	3.94	3.73	2.45	4.63	4.32
price	3.90	4.18	2.82	4.67	3.65
single_portion	2.77	3.06	1.80	4.08	2.08
sustainable_sources	3.82	3.16	2.51	4.59	4.42
sustainable_packaging	3.65	2.96	2.33	4.47	4.22
speed_n_simplicity	3.32	3.64	2.31	3.98	3.09

Cluster_KMeans	Population	1	2	3	4
freshness	4.29	4.13	2.90	4.76	4.75
origin	3.88	3.28	2.43	4.79	4.39
nutritional_balance	3.69	3.52	2.18	4.67	3.85
innovative_products	2.60	2.59	1.69	3.76	2.20
unprocessed	3.94	3.73	2.45	4.63	4.32
price	3.90	4.18	2.82	4.67	3.65
single_portion	2.77	3.06	1.80	4.08	2.08
sustainable_sources	3.82	3.16	2.51	4.59	4.42
sustainable_packaging	3.65	2.96	2.33	4.47	4.22
speed_n_simplicity	3.32	3.64	2.31	3.98	3.09

	age	region	location	gender	family_members	job	consumption_frequency	fish_type	meal_preparation_time	grocery_responsibility	grocery_place	WTP_fish
respondent_id
0	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	Una volta a settimana	Pesce surgelato	15-30 minuti	Raramente/Mai	Supermercato	Più di 10€
1	18-25 anni	Lombardia	Centro città	Uomo	2-3	Lavoratore	Una volta a settimana	Pesce surgelato	15-30 minuti	A volte	Supermercato;Negozi specializzati (es. pescherie)	5-8€
2	18-25 anni	Veneto	Centro città	Donna	1	Studente	Una volta a settimana	Pesce fresco	30-60 minuti	Sempre/Quasi sempre	Supermercato;Mercati locali	5-8€
3	26-35 anni	Lombardia	Periferia città	Donna	2-3	Lavoratore	Una o due volte al mese	Pesce fresco	30-60 minuti	Sempre/Quasi sempre	Supermercato	Non lo comprerei
4	60-70 anni	Lombardia	Esterno (es. campagna/costiera/montagna...)	Donna	1	Lavoratore	Una o due volte al mese	Pesce surgelato	15-30 minuti	Sempre/Quasi sempre	Supermercato	Meno di 5€
...	...	...	...	...	...	...	...	...	...	...	...	...
341	60-70 anni	Campania	Periferia città	Donna	2-3	Lavoratore	Una o due volte al mese	Pesce surgelato	30-60 minuti	A volte	Supermercato	Non lo comprerei
342	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	Una volta a settimana	Pesce fresco	30-60 minuti	Sempre/Quasi sempre	Supermercato	5-8€
343	26-35 anni	Lazio	Centro città	Uomo	1	Lavoratore	Una o due volte al mese	Pesce in scatola	Meno di 15 minuti	Sempre/Quasi sempre	Supermercato	Meno di 5€
344	18-25 anni	Lombardia	Centro città	Donna	4+	Studente	Una volta a settimana	Pesce surgelato	15-30 minuti	Sempre/Quasi sempre	Supermercato	Meno di 5€
345	60-70 anni	Lombardia	Esterno (es. campagna/costiera/montagna...)	Uomo	2-3	Lavoratore	Una o due volte al mese	Pesce surgelato	30-60 minuti	Sempre/Quasi sempre	Supermercato	Più di 10€

	age	region	location	gender	family_members	job	Cluster_K4
respondent_id
0	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	4
1	18-25 anni	Lombardia	Centro città	Uomo	2-3	Lavoratore	4
2	18-25 anni	Veneto	Centro città	Donna	1	Studente	1
3	26-35 anni	Lombardia	Periferia città	Donna	2-3	Lavoratore	4
4	60-70 anni	Lombardia	Esterno	Donna	1	Lavoratore	3
...	...	...	...	...	...	...	...
341	60-70 anni	Campania	Periferia città	Donna	2-3	Lavoratore	3
342	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	1
343	26-35 anni	Lazio	Centro città	Uomo	1	Lavoratore	2
344	18-25 anni	Lombardia	Centro città	Donna	4+	Studente	1
345	60-70 anni	Lombardia	Esterno	Uomo	2-3	Lavoratore	4

	age	region	location	gender	family_members	job	Cluster_K4
respondent_id
0	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	4
1	18-25 anni	Lombardia	Centro città	Uomo	2-3	Lavoratore	4
2	18-25 anni	Veneto	Centro città	Donna	1	Studente	1
3	26-35 anni	Lombardia	Periferia città	Donna	2-3	Lavoratore	4
4	60-70 anni	Lombardia	Esterno	Donna	1	Lavoratore	3
...	...	...	...	...	...	...	...
341	60-70 anni	Campania	Periferia città	Donna	2-3	Lavoratore	3
342	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	1
343	26-35 anni	Lazio	Centro città	Uomo	1	Lavoratore	2
344	18-25 anni	Lombardia	Centro città	Donna	4+	Studente	1
345	60-70 anni	Lombardia	Esterno	Uomo	2-3	Lavoratore	4

MAREPIU - AN ANALYSIS OF B2C MARKET OPPORTUNITIES¶

Table of Contents¶

1. Data Loading and Description¶

2. Segmentation¶

3. Classification¶

4. Conclusions and Limitations¶

SEGMENTATION¶

1. Data loading and description¶

2. Segmentation¶

2.1. Data¶

2.2. Clustering¶

2.2.1. Optimal number of clusters¶

2.2.2. K-Means (k=2)¶

2.2.3. K-Means (k=4)¶

2.2.4. Hierarchical Clustering¶

2.3. PCA¶

2.3.1. Principal Components¶

2.3.2. Optimal number of clusters¶

2.3.3. K-Means (k=4)¶

2.3.4. Clusters Analysis (k=2)¶

2.4. Variable selection¶

2.4.1. Optimal number of clusters.¶

2.4.2. K-Means (k=2)¶

2.4.3. K-Means (k=4)¶

2.5. Clusters Analysis¶

2.5.1. Analysis of sample¶

2.5.2. Analysis of clusters¶

3. Classification¶

3.1. Multinomial Logit model¶

3.2. Random Forest¶

4. Conclusions and Limitations¶

	Informazioni cronologiche	Qual è la tua età?	In che regione vivi?	In che zona abiti?	Con quale genere ti identifichi?	Da quanti componenti è formato il tuo nucleo familiare?	Qual è la tua professione?	Consumi prodotti ittici?	Per quale motivo non consumi prodotti ittici?	Quanto spesso consumi prodotti ittici?	...	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La velocità di preparazione]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La semplicità di preparazione]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Il bilancio nutrizionale del prodotto]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La presenza di prodotti nuovi/innovativi]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Che il prodotto sia al naturale/poco lavorato]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [Il prezzo]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche? [La presenza di prodotti monoporzione]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [La provenienza da fonti sostenibili degli ingredienti (es. pesca certificata)]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [Il basso impatto ambientale della produzione]	Da 1 a 5, quanto sono importanti per te le seguenti caratteristiche in termini di sostenibilità? [Il packaging eco-sostenibile ]
0	2024/11/24 11:38:10 AM CET	18-25 anni	Lombardia	Centro città	Uomo	4+	Studente	Si	NaN	Una volta a settimana	...	3 - Mediamente importante	2 - Poco importante	4 - Abbastanza importante	1 - Non mi interessa	2 - Poco importante	5 - Molto importante	2 - Poco importante	5 - Molto importante	4 - Abbastanza importante	4 - Abbastanza importante
1	2024/11/24 11:39:37 AM CET	18-25 anni	Lombardia	Centro città	Uomo	2-3	Lavoratore	Si	NaN	Una volta a settimana	...	4 - Abbastanza importante	4 - Abbastanza importante	3 - Mediamente importante	2 - Poco importante	4 - Abbastanza importante	4 - Abbastanza importante	4 - Abbastanza importante	5 - Molto importante	5 - Molto importante	5 - Molto importante
2	2024/11/24 11:39:51 AM CET	18-25 anni	Veneto	Centro città	Donna	1	Studente	Si	NaN	Una volta a settimana	...	3 - Mediamente importante	2 - Poco importante	5 - Molto importante	2 - Poco importante	3 - Mediamente importante	4 - Abbastanza importante	4 - Abbastanza importante	3 - Mediamente importante	3 - Mediamente importante	3 - Mediamente importante
3	2024/11/24 11:48:20 AM CET	26-35 anni	Lombardia	Periferia città	Donna	2-3	Lavoratore	Si	NaN	Una o due volte al mese	...	2 - Poco importante	3 - Mediamente importante	4 - Abbastanza importante	1 - Non mi interessa	5 - Molto importante	4 - Abbastanza importante	1 - Non mi interessa	3 - Mediamente importante	3 - Mediamente importante	3 - Mediamente importante
4	2024/11/24 11:49:38 AM CET	60-70 anni	Lombardia	Esterno (es. campagna/costiera/montagna...)	Donna	1	Lavoratore	Si	NaN	Una o due volte al mese	...	5 - Molto importante	5 - Molto importante	4 - Abbastanza importante	2 - Poco importante	5 - Molto importante	5 - Molto importante	5 - Molto importante	5 - Molto importante	5 - Molto importante	5 - Molto importante

	freshness	origin	cooking_speed	cooking_simplicity	nutritional_balance	innovative_products	unprocessed	price	single_portion	sustainable_sources	low_impact_environment	sustainable_packaging
count	308.0	308.0	308.0	308.0	308.0	308.0	308.0	308.0	308.0	308.0	308.0	308.0
mean	4.3	3.9	3.3	3.4	3.7	2.6	3.9	3.9	2.8	3.8	3.8	3.6
std	1.0	1.1	1.1	1.1	1.2	1.2	1.1	1.0	1.3	1.1	1.2	1.2
min	1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0
25%	4.0	3.0	3.0	3.0	3.0	2.0	3.0	3.0	2.0	3.0	3.0	3.0
50%	5.0	4.0	3.0	3.5	4.0	2.5	4.0	4.0	3.0	4.0	4.0	4.0
75%	5.0	5.0	4.0	4.0	5.0	3.0	5.0	5.0	4.0	5.0	5.0	5.0
max	5.0	5.0	5.0	5.0	5.0	5.0	5.0	5.0	5.0	5.0	5.0	5.0

Cluster_KMeans	Population	1	2
freshness	4.29	4.71	3.63
origin	3.88	4.52	2.88
nutritional_balance	3.69	4.26	2.81
innovative_products	2.60	2.97	2.02
unprocessed	3.94	4.48	3.09
price	3.90	4.13	3.53
single_portion	2.77	3.17	2.17
sustainable_sources	3.82	4.41	2.91
sustainable_packaging	3.65	4.27	2.68
speed_n_simplicity	3.32	3.61	2.88

Dep. Variable:	Cluster_K4	No. Observations:	306
Model:	MNLogit	Df Residuals:	264
Method:	MLE	Df Model:	39
Date:	Fri, 31 Jan 2025	Pseudo R-squ.:	0.08100
Time:	16:05:06	Log-Likelihood:	-381.81
converged:	False	LL-Null:	-415.46
Covariance Type:	nonrobust	LLR p-value:	0.003243

Cluster_K4=2	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	20.9149	1.31e+04	0.002	0.999	-2.57e+04	2.57e+04
age_26_35_anni	-0.5837	0.636	-0.918	0.358	-1.829	0.662
age_36_45_anni	-0.2881	1.133	-0.254	0.799	-2.509	1.933
age_46_60_anni	-0.5617	0.590	-0.952	0.341	-1.719	0.595
age_60_70_anni	-0.1418	1.146	-0.124	0.902	-2.389	2.105
age_70__anni	-14.8100	1.99e+05	-7.45e-05	1.000	-3.9e+05	3.9e+05
family_members_2_3	0.2648	0.613	0.432	0.666	-0.937	1.467
family_members_4_	0.2489	0.621	0.401	0.689	-0.969	1.467
gender_Uomo	0.1500	0.391	0.384	0.701	-0.616	0.916
job_Lavoratore	-21.0043	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
job_Pensionato	1.5493	3.64e+04	4.26e-05	1.000	-7.13e+04	7.13e+04
job_Studente	-21.3917	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
location_Esterno	-0.5047	0.527	-0.957	0.339	-1.539	0.529
location_Periferia_città	-0.6791	0.427	-1.591	0.112	-1.516	0.157
Cluster_K4=3	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	20.2459	1.31e+04	0.002	0.999	-2.57e+04	2.57e+04
age_26_35_anni	-0.2072	0.597	-0.347	0.728	-1.377	0.962
age_36_45_anni	0.4767	0.986	0.484	0.629	-1.455	2.409
age_46_60_anni	0.2495	0.521	0.479	0.632	-0.772	1.271
age_60_70_anni	1.2021	0.904	1.329	0.184	-0.571	2.975
age_70__anni	3.3001	1.99e+05	1.66e-05	1.000	-3.9e+05	3.9e+05
family_members_2_3	0.5342	0.559	0.955	0.340	-0.562	1.630
family_members_4_	0.2902	0.583	0.498	0.619	-0.852	1.433
gender_Uomo	-0.9145	0.415	-2.203	0.028	-1.728	-0.101
job_Lavoratore	-20.2145	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
job_Pensionato	1.5043	3.64e+04	4.13e-05	1.000	-7.13e+04	7.13e+04
job_Studente	-21.3325	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
location_Esterno	-0.3896	0.501	-0.777	0.437	-1.372	0.593
location_Periferia_città	-0.3196	0.399	-0.801	0.423	-1.101	0.462
Cluster_K4=4	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	19.2575	1.31e+04	0.001	0.999	-2.57e+04	2.57e+04
age_26_35_anni	-0.1062	0.537	-0.198	0.843	-1.159	0.946
age_36_45_anni	0.4970	0.935	0.531	0.595	-1.336	2.330
age_46_60_anni	-0.1409	0.497	-0.284	0.777	-1.115	0.833
age_60_70_anni	1.2257	0.868	1.412	0.158	-0.475	2.927
age_70__anni	4.0272	1.99e+05	2.03e-05	1.000	-3.9e+05	3.9e+05
family_members_2_3	1.2102	0.589	2.056	0.040	0.057	2.364
family_members_4_	1.1403	0.602	1.893	0.058	-0.041	2.321
gender_Uomo	-0.0269	0.329	-0.082	0.935	-0.672	0.618
job_Lavoratore	-19.9809	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
job_Pensionato	1.4601	3.64e+04	4.01e-05	1.000	-7.13e+04	7.13e+04
job_Studente	-20.2648	1.31e+04	-0.002	0.999	-2.57e+04	2.57e+04
location_Esterno	0.2544	0.429	0.594	0.553	-0.586	1.094
location_Periferia_città	-0.3849	0.371	-1.039	0.299	-1.111	0.341

Cluster_K4=2	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	14.5563	5.25e+06	2.77e-06	1.000	-1.03e+07	1.03e+07
family_members_2_3	0.3820	0.588	0.650	0.516	-0.770	1.534
family_members_4_	0.4086	0.589	0.694	0.488	-0.745	1.563
gender_Donna	4.9261	5.25e+06	9.39e-07	1.000	-1.03e+07	1.03e+07
gender_Uomo	5.0758	5.25e+06	9.67e-07	1.000	-1.03e+07	1.03e+07
job_Lavoratore	-20.0735	8689.005	-0.002	0.998	-1.71e+04	1.7e+04
job_Pensionato	4.0200	6.71e+04	5.99e-05	1.000	-1.31e+05	1.31e+05
job_Studente	-20.1453	8689.005	-0.002	0.998	-1.71e+04	1.7e+04
location_Esterno	-0.4532	0.520	-0.872	0.383	-1.472	0.565
location_Periferia_città	-0.6610	0.421	-1.571	0.116	-1.486	0.164
Cluster_K4=3	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	45.9883	4.66e+05	9.88e-05	1.000	-9.13e+05	9.13e+05
family_members_2_3	0.6091	0.541	1.126	0.260	-0.451	1.669
family_members_4_	0.3414	0.560	0.610	0.542	-0.756	1.439
gender_Donna	-26.4073	4.66e+05	-5.67e-05	1.000	-9.12e+05	9.12e+05
gender_Uomo	-27.4071	4.66e+05	-5.89e-05	1.000	-9.12e+05	9.12e+05
job_Lavoratore	-19.4114	8689.005	-0.002	0.998	-1.7e+04	1.7e+04
job_Pensionato	4.8034	6.71e+04	7.16e-05	1.000	-1.31e+05	1.31e+05
job_Studente	-20.7654	8689.005	-0.002	0.998	-1.71e+04	1.7e+04
location_Esterno	-0.2695	0.491	-0.549	0.583	-1.232	0.693
location_Periferia_città	-0.3169	0.394	-0.805	0.421	-1.088	0.455
Cluster_K4=4	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	44.4412	4.66e+05	9.54e-05	1.000	-9.13e+05	9.13e+05
family_members_2_3	1.2387	0.565	2.194	0.028	0.132	2.345
family_members_4_	1.1137	0.571	1.949	0.051	-0.006	2.234
gender_Donna	-25.9750	4.66e+05	-5.58e-05	1.000	-9.12e+05	9.12e+05
gender_Uomo	-26.0813	4.66e+05	-5.6e-05	1.000	-9.12e+05	9.12e+05
job_Lavoratore	-19.1123	8689.005	-0.002	0.998	-1.7e+04	1.7e+04
job_Pensionato	5.1422	6.71e+04	7.67e-05	1.000	-1.31e+05	1.31e+05
job_Studente	-19.4128	8689.005	-0.002	0.998	-1.7e+04	1.7e+04
location_Esterno	0.2820	0.419	0.673	0.501	-0.539	1.103
location_Periferia_città	-0.4191	0.365	-1.149	0.251	-1.134	0.296

Cluster_K4=2	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	11.6503	2470.149	0.005	0.996	-4829.753	4853.054
family_members_2_3	0.1853	0.572	0.324	0.746	-0.935	1.306
family_members_4_	0.2334	0.573	0.407	0.684	-0.891	1.357
gender_Donna	2.9196	2334.959	0.001	0.999	-4573.516	4579.355
gender_Uomo	3.0355	2334.959	0.001	0.999	-4573.400	4579.471
job_Lavoratore	-15.3707	805.980	-0.019	0.985	-1595.063	1564.322
job_Pensionato	6.5555	2.4e+04	0.000	1.000	-4.7e+04	4.7e+04
job_Studente	-15.3982	805.980	-0.019	0.985	-1595.091	1564.294
Cluster_K4=3	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	27.1135	899.166	0.030	0.976	-1735.219	1789.446
family_members_2_3	0.5280	0.530	0.996	0.319	-0.511	1.567
family_members_4_	0.2561	0.550	0.466	0.641	-0.822	1.334
gender_Donna	-12.3886	398.616	-0.031	0.975	-793.662	768.885
gender_Uomo	-13.3990	398.616	-0.034	0.973	-794.673	767.875
job_Lavoratore	-14.6841	805.980	-0.018	0.985	-1594.376	1565.008
job_Pensionato	7.4330	2.4e+04	0.000	1.000	-4.7e+04	4.7e+04
job_Studente	-16.0152	805.980	-0.020	0.984	-1595.708	1563.677
Cluster_K4=4	coef	std err	z	P>\|z\|	[0.025	0.975]
Intercept	25.4079	899.166	0.028	0.977	-1736.925	1787.741
family_members_2_3	1.1617	0.550	2.114	0.035	0.085	2.239
family_members_4_	1.0774	0.556	1.939	0.052	-0.012	2.166
gender_Donna	-11.7737	398.616	-0.030	0.976	-793.047	769.499
gender_Uomo	-11.9175	398.616	-0.030	0.976	-793.191	769.356
job_Lavoratore	-14.3280	805.980	-0.018	0.986	-1594.021	1565.364
job_Pensionato	7.7135	2.4e+04	0.000	1.000	-4.7e+04	4.7e+04
job_Studente	-14.6026	805.980	-0.018	0.986	-1594.295	1565.090

	Cluster	Count	Percentage
0	1	187	60.714286
1	2	121	39.285714

	Cluster	Count	Percentage
0	1	79	25.649351
1	2	51	16.558442
2	3	75	24.350649
3	4	103	33.441558